from concurrent.futures import ThreadPoolExecutor, as_completed
import os
from concurrent_study.blog_spider import urls, craw
from concurrent_study.common import cal_timer

cpu_count = os.cpu_count()
print(f"Number of CPUs: {cpu_count}")

with ThreadPoolExecutor(max_workers=cpu_count) as executor:
    future_to_url = {executor.submit(craw, url): url for url in urls}
    # 等待所有线程完成并获取结果
    for future in as_completed(future_to_url):
        url = future_to_url[future]
        try:
            data = future.result()
            print(f"{url} page is {len(data)} bytes")
        except Exception as exc:
            print(f"{url} generated an exception: {exc}")
